{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-3-a18bf5c6af71>:50: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From C:\\Users\\chenxi\\AppData\\Roaming\\Python\\Python36\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From C:\\Users\\chenxi\\AppData\\Roaming\\Python\\Python36\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting C:/Users/chenxi/Desktop/MNIST_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Users\\chenxi\\AppData\\Roaming\\Python\\Python36\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting C:/Users/chenxi/Desktop/MNIST_data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Users\\chenxi\\AppData\\Roaming\\Python\\Python36\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting C:/Users/chenxi/Desktop/MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting C:/Users/chenxi/Desktop/MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Users\\chenxi\\AppData\\Roaming\\Python\\Python36\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From <ipython-input-3-a18bf5c6af71>:94: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See @{tf.nn.softmax_cross_entropy_with_logits_v2}.\n",
      "\n",
      "step 100, entropy loss: 0.245699, l2_loss: 287.069427, total loss: 0.257181\n",
      "0.9341\n",
      "step 200, entropy loss: 0.094516, l2_loss: 397.994263, total loss: 0.110436\n",
      "0.9483\n",
      "step 300, entropy loss: 0.212120, l2_loss: 490.875183, total loss: 0.231755\n",
      "0.949\n",
      "step 400, entropy loss: 0.133408, l2_loss: 565.429138, total loss: 0.156025\n",
      "0.958\n",
      "step 500, entropy loss: 0.205770, l2_loss: 635.584167, total loss: 0.231194\n",
      "0.9553\n",
      "step 600, entropy loss: 0.189705, l2_loss: 703.372559, total loss: 0.217840\n",
      "0.9607\n",
      "step 700, entropy loss: 0.074672, l2_loss: 701.226074, total loss: 0.102721\n",
      "0.9715\n",
      "step 800, entropy loss: 0.090861, l2_loss: 693.062683, total loss: 0.118583\n",
      "0.9718\n",
      "step 900, entropy loss: 0.036899, l2_loss: 691.999634, total loss: 0.064579\n",
      "0.97\n",
      "step 1000, entropy loss: 0.094053, l2_loss: 695.094788, total loss: 0.121856\n",
      "0.9729\n",
      "step 1100, entropy loss: 0.037551, l2_loss: 703.539368, total loss: 0.065692\n",
      "0.9739\n",
      "step 1200, entropy loss: 0.059830, l2_loss: 705.536987, total loss: 0.088052\n",
      "0.9721\n",
      "step 1300, entropy loss: 0.052092, l2_loss: 696.989258, total loss: 0.079971\n",
      "0.9775\n",
      "step 1400, entropy loss: 0.064934, l2_loss: 689.326599, total loss: 0.092507\n",
      "0.976\n",
      "step 1500, entropy loss: 0.078745, l2_loss: 686.897827, total loss: 0.106221\n",
      "0.976\n",
      "step 1600, entropy loss: 0.038281, l2_loss: 682.915527, total loss: 0.065598\n",
      "0.9747\n",
      "step 1700, entropy loss: 0.023262, l2_loss: 681.160889, total loss: 0.050508\n",
      "0.9787\n",
      "step 1800, entropy loss: 0.029413, l2_loss: 673.766479, total loss: 0.056364\n",
      "0.9795\n",
      "step 1900, entropy loss: 0.029753, l2_loss: 665.374878, total loss: 0.056368\n",
      "0.9783\n",
      "step 2000, entropy loss: 0.063195, l2_loss: 659.666992, total loss: 0.089582\n",
      "0.9802\n",
      "step 2100, entropy loss: 0.051829, l2_loss: 654.780396, total loss: 0.078020\n",
      "0.9808\n",
      "step 2200, entropy loss: 0.006229, l2_loss: 650.074036, total loss: 0.032232\n",
      "0.9792\n",
      "step 2300, entropy loss: 0.028430, l2_loss: 643.200256, total loss: 0.054158\n",
      "0.9804\n",
      "step 2400, entropy loss: 0.013570, l2_loss: 637.485840, total loss: 0.039070\n",
      "0.9795\n",
      "step 2500, entropy loss: 0.009796, l2_loss: 633.254944, total loss: 0.035126\n",
      "0.9814\n",
      "step 2600, entropy loss: 0.021593, l2_loss: 628.853333, total loss: 0.046747\n",
      "0.9812\n",
      "step 2700, entropy loss: 0.016734, l2_loss: 624.306152, total loss: 0.041706\n",
      "0.9806\n",
      "step 2800, entropy loss: 0.018209, l2_loss: 619.868713, total loss: 0.043004\n",
      "0.9815\n",
      "step 2900, entropy loss: 0.022062, l2_loss: 615.055420, total loss: 0.046664\n",
      "0.982\n",
      "step 3000, entropy loss: 0.026327, l2_loss: 612.122864, total loss: 0.050812\n",
      "0.9805\n"
     ]
    }
   ],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "import numpy as np\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None\n",
    "\n",
    "\n",
    "def swish(x):\n",
    "    return x * tf.nn.sigmoid(x)\n",
    "\n",
    "\n",
    "def selu(x):\n",
    "    with tf.name_scope('elu') as scope:\n",
    "        alpha = 1.6732632423543772848170429916717\n",
    "        scale = 1.0507009873554804934193349852946\n",
    "        return scale * tf.where(x >= 0.0, x, alpha * tf.nn.elu(x))\n",
    "\n",
    "\n",
    "def relu(x):\n",
    "    return tf.nn.relu(x)\n",
    "\n",
    "\n",
    "def activation(x):\n",
    "    #  return selu(x)\n",
    "    #  return relu(x)\n",
    "    #  return tf.nn.sigmoid(x)\n",
    "    #  return tf.nn.elu(x)\n",
    "    return swish(x)\n",
    "\n",
    "\n",
    "def initialize(shape, stddev=0.1):\n",
    "    return tf.truncated_normal(shape, stddev=stddev)\n",
    "    # return tf.zeros(shape)\n",
    "\n",
    "\n",
    "# Import data\n",
    "data_dir = 'C:/Users/chenxi/Desktop/MNIST_data/'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)\n",
    "\n",
    "init_learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "# Create the model\n",
    "L1_units_count = 100\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "# tf.shape(x)  [100, 784]\n",
    "# exponetial lr decay\n",
    "epoch_steps = tf.to_int64(tf.div(60000, tf.shape(x)[0]))\n",
    "global_step = tf.train.get_or_create_global_step()\n",
    "current_epoch = global_step // epoch_steps\n",
    "decay_times = current_epoch\n",
    "current_learning_rate = tf.multiply(init_learning_rate,\n",
    "                                    tf.pow(0.575, tf.to_float(decay_times)))\n",
    "\n",
    "W_1 = tf.Variable(initialize([784, L1_units_count],\n",
    "                             stddev=np.sqrt(2 / 784)))\n",
    "b_1 = tf.Variable(tf.constant(0.001, shape=[L1_units_count]))\n",
    "logits_1 = tf.matmul(x, W_1) + b_1\n",
    "output_1 = activation(logits_1)\n",
    "\n",
    "L2_units_count = 10\n",
    "W_2 = tf.Variable(initialize([L1_units_count,\n",
    "                              L2_units_count],\n",
    "                             stddev=np.sqrt(2 / L1_units_count)))\n",
    "b_2 = tf.Variable(tf.constant(0.001, shape=[L2_units_count]))\n",
    "logits_2 = tf.matmul(output_1, W_2) + b_2\n",
    "\n",
    "y = logits_2\n",
    "\n",
    "# Define loss and optimizer\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "l2_loss = tf.nn.l2_loss(W_1) + tf.nn.l2_loss(W_2)\n",
    "total_loss = cross_entropy + 4e-5 * l2_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(current_learning_rate)\n",
    "gradients = optimizer.compute_gradients(total_loss)\n",
    "train_step = optimizer.apply_gradients(gradients)\n",
    "\n",
    "train_step = tf.train.AdamOptimizer(\n",
    "    current_learning_rate).minimize(\n",
    "    total_loss, global_step=global_step)\n",
    "# weight decay\n",
    "# 0 0 0 0 1 0 0 0 0\n",
    "# 0 1 0 0 0 0 0 0 0\n",
    "# 还记得不？不要把graph定义写到图里面\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "tf.global_variables_initializer().run()\n",
    "# Train\n",
    "for step in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    lr = 1e-2\n",
    "    _, loss, l2_loss_value, total_loss_value, current_lr_value = \\\n",
    "        sess.run(\n",
    "            [train_step, cross_entropy, l2_loss, total_loss,\n",
    "             current_learning_rate],\n",
    "            feed_dict={x: batch_xs, y_: batch_ys,\n",
    "                       init_learning_rate: lr})\n",
    "\n",
    "    if (step + 1) % 100 == 0:\n",
    "        print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' %\n",
    "              (step + 1, loss, l2_loss_value, total_loss_value))\n",
    "        # print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys}))\n",
    "        print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                            y_: mnist.test.labels}))\n",
    "        # print(current_lr_value)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
